"""Manage GitLab runners and project variables."""

import argparse
import base64
import contextlib
import copy
import difflib
import functools
from functools import cached_property
import itertools
import os
import pathlib
import re
import subprocess
import sys
import tempfile
from urllib.parse import urlparse

from cki_lib import config_tree
from cki_lib import misc
from cki_lib import yaml
from cki_lib.logger import get_logger
from cki_lib.session import get_session
from gitlab import Gitlab
from gitlab.exceptions import GitlabGetError
from gitlab.v4.objects import ProjectHookManager
import sentry_sdk
import toml

from cki.cki_tools import _utils
from cki_tools.credentials import secrets

LOGGER = get_logger('cki_tools.gitlab_runner_config')
SESSION = get_session('cki_tools.gitlab_runner_config')


class GroupHookManager(ProjectHookManager):
    # pylint: disable=too-many-ancestors
    """Hack python-gitlab to provide group hook support."""

    _path = "/groups/%(group_id)s/hooks"
    _from_parent_attrs = {"group_id": "id"}


# pylint: disable=too-many-instance-attributes
class RunnerConfig:
    """Manage GitLab runners and project variables."""

    def __init__(self, config, args):
        """Initialize and load the config."""
        self.args = args

        # https://rednafi.github.io/reflections/dont-wrap-instance-methods-with-functoolslru_cache-decorator-in-python.html
        self.webhook_for_routing_key = functools.lru_cache()(self._webhook_for_routing_key)

        self.runner_tokens = config.get('runner_tokens', {})
        self.gitlab_instances = config_tree.process_config_tree(
            config.get('gitlab_instances', {}))
        self.runner_templates = config_tree.process_config_tree(
            config.get('runner_templates', {}))
        self.runner_deployments = config_tree.process_config_tree(
            config.get('runner_deployments', {}))
        self.variable_groups = config_tree.process_config_tree(
            config.get('variable_groups', {}))
        self.runner_configurations = config_tree.process_config_tree(
            config.get('runner_configurations', {}))
        self.gitlab_variables = config.get('gitlab_variables', [])
        self.webhook_endpoints = config.get('webhook_endpoints', [])
        self.token_not_found = False
        self._group_cache = {}
        self._project_cache = {}

    @staticmethod
    @contextlib.contextmanager
    def _tempfile(prefix, contents):
        """Create a temporary yaml file and return the name."""
        with tempfile.NamedTemporaryFile(mode='w+t',
                                         prefix=prefix,
                                         suffix='.yaml') as temp:
            temp.write(contents)
            temp.flush()
            yield temp.name

    @staticmethod
    def _sorted(value):
        """Recursivly sort a data structure."""
        if isinstance(value, dict):
            return {k: RunnerConfig._sorted(v)
                    for k, v in sorted(value.items())}
        if isinstance(value, list):
            return [RunnerConfig._sorted(v) for v in value]
        return value

    @staticmethod
    def _list(dictionary, key):
        """Return list or a one-element list for single strings."""
        return list(misc.flattened(dictionary.get(key, [])))

    @staticmethod
    def _gitlab_variables(gl_project_group):
        """Return a dict of project/group CI/CD variables."""
        return {v.key: v.value
                for v in gl_project_group.variables.list(iterator=True)}

    def _gitlab_runner(self, instance_name, runner_id):
        """Return a dict describing a gitlab runner config."""
        gl_instance = self._gl_instance(instance_name)
        gl_runner = gl_instance.runners.get(runner_id)
        runner = {
            '.groups': [],
            '.projects': [],
            '.id': gl_runner.id,
            '.paused': gl_runner.paused,
            'description': gl_runner.description,
            'access_level': gl_runner.access_level,
            'locked': gl_runner.locked,
            'maximum_timeout': gl_runner.maximum_timeout,
            'run_untagged': gl_runner.run_untagged,
            'tag_list': sorted(gl_runner.tag_list),
        }
        for group in gl_runner.groups:
            gid = group['id']
            if gid not in self._group_cache:
                self._group_cache[gid] = gl_instance.groups.get(gid)
            path = self._group_cache[gid].full_path
            runner['.groups'].append(f'{instance_name}/{path}')
        for project in gl_runner.projects:
            pid = project['id']
            if pid not in self._project_cache:
                self._project_cache[pid] = gl_instance.projects.get(pid)
            path = self._project_cache[pid].path_with_namespace
            runner['.projects'].append(f'{instance_name}/{path}')
        runner['.groups'] = sorted(runner['.groups'])
        runner['.projects'] = sorted(runner['.projects'])
        return runner

    @cached_property
    def _gitlab_runner_overview(self):
        """Return a dict describing all accessible gitlab runners.

        This list contains very few fields, but should be enough to figure out
        the status of the runners.
        """
        runners = {}
        for instance_name in self.gitlab_instances:
            gl_instance = self._gl_instance(instance_name)
            for gl_runner in gl_instance.runners.list(iterator=True):
                runner = {
                    '.instance_name': instance_name,
                    '.online': gl_runner.online or False,  # might be null
                    'id': gl_runner.id,
                    'paused': gl_runner.paused,
                }
                runners[gl_runner.description] = runner
        return runners

    @cached_property
    def _gitlab_runners(self):
        """Return a dict describing all accessible gitlab runners."""
        runners = {}
        for description, runner in self._gitlab_runner_overview.items():
            print(f'Getting details for {description}', file=sys.stderr)
            runners[description] = self._gitlab_runner(runner['.instance_name'], runner['id'])
        return runners

    def _gl_instance(self, instance_name):
        """Return a Gitlab instance for an instance abbreviation."""
        config = self.gitlab_instances[instance_name]
        return Gitlab(config['url'], config['api_token'], session=SESSION)

    def _gl_group(self, group_name, lazy=True):
        """Return a Gitlab instance for an instance/group."""
        instance_name, group = group_name.split('/', 1)
        gl_instance = self._gl_instance(instance_name)
        gl_group = gl_instance.groups.get(group, lazy=lazy)
        gl_group.hooks = GroupHookManager(gl_group.manager.gitlab, gl_group)
        return gl_group

    def _gl_project(self, project_name, lazy=True):
        """Return a Gitlab instance for an instance/project."""
        instance_name, project = project_name.split('/', 1)
        gl_instance = self._gl_instance(instance_name)
        return gl_instance.projects.get(project, lazy=lazy)

    def _registration_data(self, group_names, project_names):
        """Try to find instance+registration token for a list of projects."""
        return next((
            {'instance': n.split('/', 1)[0], 'token': token}
            for (pg, n) in itertools.chain(
                ((self._gl_project(p, lazy=False), p) for p in project_names),
                ((self._gl_group(g, lazy=False), g) for g in group_names),
            ) if (token := pg.attributes.get('runners_token'))
        ), None)

    @staticmethod
    def _get_instance_name(name, project_group_names):
        """Return the common instance abbreviation for a list of projects."""
        instance_names = [p.split('/', 1)[0] for p in project_group_names]
        if not instance_names:
            raise Exception(f'Cannot find instance for {name}')
        if any(instance_names[0] != u for u in instance_names):
            raise Exception(f'Multiple instances for {name}')
        return instance_names[0]

    def _get_instance_url(self, configuration_name):
        """Return the common instance url for a configuration."""
        configuration = self.runner_configurations[configuration_name]
        project_group_names = (self._list(configuration, 'gitlab_projects') +
                               self._list(configuration, 'gitlab_groups'))
        return self.gitlab_instances[self._get_instance_name(
            configuration_name, project_group_names)]['url']

    @staticmethod
    def _get_cache_section(bucket_spec, configuration_name):
        """Return the cache section of a gitlab-runner config file."""
        spec = _utils.parse_bucket_spec(bucket_spec)
        endpoint = urlparse(spec.endpoint)
        return {
            'Type': 's3',
            'Shared': True,
            'Path': spec.prefix + configuration_name,
            's3': {
                'ServerAddress': endpoint.hostname,
                'AccessKey': spec.access_key,
                'SecretKey': spec.secret_key,
                'BucketName': spec.bucket,
                'Insecure': endpoint.scheme == 'http',
            }}

    def _configuration_runners(self, configuration_name):
        """Return all runners for a configuration."""
        configuration = self.runner_configurations[configuration_name]
        runner_deployments = configuration['runner_deployments']
        runners = []
        for deployment_name in runner_deployments:
            runners += self._deployment_runners(
                configuration_name, deployment_name,
                self._list(runner_deployments, deployment_name),
                configuration.get('paused', False))
        return runners

    def _deployment_runners(self, configuration_name, deployment_name,
                            runner_template_names, paused):
        # pylint: disable=too-many-locals
        """Return all runners for a deployment of a configuration."""
        runners = []
        configuration = self.runner_configurations[configuration_name]
        deployment = self.runner_deployments[deployment_name]
        for runner_name in runner_template_names:
            runner_config = copy.deepcopy(self.runner_templates[runner_name])
            config_tree.merge_dicts(runner_config,
                                    deployment.get('.template_overrides', {}))
            runner_config['.deployment_name'] = deployment_name
            cache = runner_config.get('.cache')
            if cache:
                runner_config['cache'] = self._get_cache_section(
                    cache, configuration_name)
            machine_options = misc.get_nested_key(
                runner_config, 'machine/.MachineOptions')
            amazonec2_tags = misc.get_nested_key(
                runner_config, 'machine/.MachineOptions/.amazonec2-tags')
            if amazonec2_tags:
                machine_options['amazonec2-tags'] = ','.join(
                    f'{k},{v}' for k, v in sorted(amazonec2_tags.items())
                )
            if machine_options:
                runner_config['machine']['MachineOptions'] = [
                    f'{k}={vv}'
                    for k, v in sorted(machine_options.items())
                    for vv in misc.flattened(v)
                    if not k.startswith('.')
                ]
            environment = {}
            variable_group = configuration.get('variable_group')
            if variable_group:
                environment.update(self.variable_groups[variable_group])
            environment.update(runner_config.get('.environment', {}))
            if environment:
                runner_config['environment'] = ['='.join(v) for v in sorted(environment.items())]
            runner_config['url'] = self._get_instance_url(configuration_name)
            fullname = f'{configuration_name}-{deployment_name}-{runner_name}'
            runner_config['name'] = fullname
            if fullname in self.runner_tokens:
                runner_config['token'] = self.runner_tokens[fullname]
            else:
                print(f'Runner token for {fullname} not found!',
                      file=sys.stderr)
                self.token_not_found = True
            if any(re.fullmatch(r, fullname) for r in self.args.activate):
                paused = False
            if any(re.fullmatch(r, fullname) for r in self.args.deactivate):
                paused = True
            runner_config['.paused'] = paused
            runners.append(runner_config)
        return runners

    @staticmethod
    def _load_config(config):
        """Load a gitlab-runner config file from a string."""
        return toml.loads(config)

    def _load_secret(self, filename):
        """Load a gitlab-runner config file from a Secret if possible."""
        path = pathlib.Path(self.args.directory, filename)
        if not path.exists():
            return {}
        secret = yaml.load(file_path=path)
        metadata_keys_to_keep = ('labels', 'name')
        metadata = {k: v for k, v in secret.get('metadata', {}).items()
                    if k in metadata_keys_to_keep}
        first_name, first_config = next(iter(secret['data'].items()))
        current = {'.metadata': metadata, '.filename': first_name}
        current.update(RunnerConfig._load_config(base64.b64decode(first_config).decode('utf8')))
        return current

    def _load_configfile(self, filename):
        """Load a gitlab-runner config file from a file if possible."""
        path = pathlib.Path(self.args.directory, filename)
        if not path.exists():
            return {}
        return RunnerConfig._load_config(path.read_text(encoding='utf8'))

    @staticmethod
    def _dump_config(config):
        """Dump a gitlab-runner config file to a string."""
        config = config_tree.clean_config(config)
        config = RunnerConfig._sorted(config)
        config['runners'] = sorted(config.get('runners', []),
                                   key=lambda r: r['name'])
        return toml.dumps(config)

    @staticmethod
    def _dump_secret(config):
        """Dump a gitlab-runner config file to a Secret."""
        return yaml.dump({
            'apiVersion': 'v1',
            'kind': 'Secret',
            'metadata': config.get('.metadata', {}),
            'stringData': {
                config.get('.filename'): RunnerConfig._dump_config(config)
            }
        }, sort_keys=False)

    def _dump_configurations(self):
        """Perform a 'configurations dump' command."""
        current = {}
        for deployment_config in self.runner_deployments.values():
            secret = deployment_config.get('.secret')
            configfile = deployment_config.get('.configfile')
            if secret:
                current[secret] = self._dump_secret(self._load_secret(secret))
            else:
                current[configfile] = self._dump_config(
                    self._load_configfile(configfile))
        return self._sorted(current)

    def _generate_configurations(self):
        """Perform a 'configurations generate' command."""
        proposed = {}
        runners = []
        for configuration_name in self.runner_configurations:
            runners += self._configuration_runners(configuration_name)
        for name, deployment_config in self.runner_deployments.items():
            deployment_config = copy.deepcopy(deployment_config)
            deployment_config['runners'] = sorted(
                (r for r in runners if r['.deployment_name'] == name),
                key=lambda r: r['name'])
            secret = deployment_config.get('.secret')
            configfile = deployment_config.get('.configfile')
            if secret:
                proposed[secret] = self._dump_secret(deployment_config)
            else:
                proposed[configfile] = self._dump_config(deployment_config)
        return self._sorted(proposed)

    def _apply_configurations(self):
        """Perform a 'configurations apply' command."""
        current = self._dump_configurations()
        proposed = self._generate_configurations()

        if current == proposed:
            print('configurations: nothing to do', file=sys.stderr)
            return

        if self.token_not_found and not self.args.dry_run:
            raise Exception('Cannot apply configurations with missing tokens')

        for name, config in proposed.items():
            print(f'{name}: updating configuration', file=sys.stderr)
            if not self.args.dry_run:
                pathlib.Path(self.args.directory, name).write_text(config, encoding='utf8')

    def _dump_registrations(self, clean=True):
        """Perform a 'registrations dump' command."""
        proposed_runners = self._generate_registrations().keys()
        groups = set()
        projects = set()
        for configuration in self.runner_configurations.values():
            groups.update(self._list(configuration, 'gitlab_groups'))
            projects.update(self._list(configuration, 'gitlab_projects'))

        result = self._sorted({name: runner
                               for name, runner in self._gitlab_runners.items()
                               if name in proposed_runners or
                               not groups.isdisjoint(runner['.groups']) or
                               not projects.isdisjoint(runner['.projects'])})
        if clean:
            for runner in result.values():
                runner.pop('.id', None)
                runner.pop('.paused', None)
        return result

    def _generate_registrations(self, clean=True):
        """Perform a 'registrations generate' command."""
        proposed = {}
        for configuration_name, configuration in self.runner_configurations.items():
            groups = self._list(configuration, 'gitlab_groups')
            projects = self._list(configuration, 'gitlab_projects')
            for runner in self._configuration_runners(configuration_name):
                proposed[runner['name']] = {
                    '.configuration_name': configuration_name,
                    '.groups': sorted(groups),
                    '.projects': sorted(projects),
                    '.paused': runner['.paused'],
                    'description': runner['name'],
                    'access_level': runner.get('.access_level',
                                               'not_protected'),
                    'locked': runner.get('.locked', True),
                    'maximum_timeout': runner.get('.maximum_timeout',
                                                  24 * 60 * 60 * 7),
                    'run_untagged': runner.get('.run_untagged', False),
                    'tag_list': sorted(self._list(runner, '.tag_list')),
                }
        if clean:
            for runner in proposed.values():
                runner.pop('.paused', None)
                runner.pop('.configuration_name', None)
        return self._sorted(proposed)

    def _apply_registration(self, name, current_registration,
                            proposed_registration):
        # pylint: disable=too-many-branches,too-many-statements
        """Perform part of a 'registrations apply' command."""
        groups = proposed_registration['.groups']
        projects = proposed_registration['.projects']
        if not current_registration:
            if not self.args.create_missing:
                raise Exception(f'Missing runner registration {name} not '
                                'created without --create-missing')
            if self.args.dry_run:
                print(f'Dry run, would create registration {name}', file=sys.stderr)
                return

            print(f'{name}: creating registration {name}', file=sys.stderr)
            proposed_payload = config_tree.clean_config(proposed_registration)
            proposed_payload['paused'] = proposed_registration['.paused']
            if groups:
                instance = groups[0].split('/', 1)[0]
                gl_group = self._gl_group(groups[0], lazy=False)
                gl_instance = gl_group.manager.gitlab
                proposed_payload['runner_type'] = 'group_type'
                proposed_payload['group_id'] = gl_group.id
            else:
                instance = projects[0].split('/', 1)[0]
                gl_project = self._gl_project(projects[0], lazy=False)
                gl_instance = gl_project.manager.gitlab
                proposed_payload['runner_type'] = 'project_type'
                proposed_payload['project_id'] = gl_project.id
            gl_instance.auth()  # materialize gl_instance.user 🤦
            runner = gl_instance.user.runners.create(proposed_payload)
            print(f'NEW RUNNER AUTH TOKEN: {name}: {runner.token}')
            with misc.only_log_exceptions():
                secrets.edit(f'gitlab-runner-token-{name}#', {
                    'token_type': 'gitlab_runner_authentication_token',
                    'instance_url': gl_instance.url,
                    'token_id': runner.id,
                })
                secrets.edit(f'gitlab-runner-token-{name}', runner.token)
            current_registration = self._gitlab_runner(instance, runner.id)
        else:
            current_payload = config_tree.clean_config(current_registration)
            proposed_payload = config_tree.clean_config(proposed_registration)
            if current_payload != proposed_payload:
                print(f'{name}: updating registration {name}', file=sys.stderr)
                if not self.args.dry_run:
                    instance = self._get_instance_name(name, groups + projects)
                    gl_instance = self._gl_instance(instance)
                    runner = gl_instance.runners.update(
                        current_registration['.id'], proposed_payload)

        if groups != current_registration['.groups']:
            raise Exception(f'Cannot change group for {name}')
        for project in set(projects) - set(current_registration['.projects']):
            print(f'{name}: enabling project {project}', file=sys.stderr)
            if not self.args.dry_run:
                instance = self._get_instance_name(name, [project])
                gl_instance = self._gl_instance(instance)
                if proposed_registration['locked']:
                    gl_instance.runners.update(
                        current_registration['.id'], {'locked': False})
                self._gl_project(project).runners.create({
                    'runner_id': current_registration['.id']})
                if proposed_registration['locked']:
                    gl_instance.runners.update(
                        current_registration['.id'], {'locked': True})
        for project in set(current_registration['.projects']) - set(projects):
            print(f'{name}: disabling project {project}', file=sys.stderr)
            if not self.args.dry_run:
                self._gl_project(project).runners.delete(
                    current_registration['.id'])

    def _apply_registrations(self):
        """Perform a 'registrations apply' command."""
        current = self._dump_registrations(clean=False)
        proposed = self._generate_registrations(clean=False)

        for name, proposed_registration in proposed.items():
            current_registration = current.get(name)
            self._apply_registration(name, current_registration,
                                     proposed_registration)

    @staticmethod
    def _merge_activation(activations, name, field, value):
        if name not in activations:
            activations[name] = {}
        if field not in activations[name]:
            activations[name][field] = value
        if activations[name][field] != value:
            activations[name][field] = 'partial'

    def _dump_activations(self):
        """Perform an 'activations dump' command."""
        return {
            name: {
                'paused': self._gitlab_runner_overview[name]['paused'],
                '.online': self._gitlab_runner_overview[name]['.online'],
            }
            for name in self._generate_registrations(clean=False)
            if name in self._gitlab_runner_overview
        }

    def _generate_activations(self):
        """Perform an 'activations generate' command."""
        return {
            name: {
                'paused': proposed_runner['.paused'],
                '.online': True,
            }
            for name, proposed_runner in self._generate_registrations(clean=False).items()
        }

    def _apply_activations(self):
        """Perform an 'activations apply' command."""
        for runner_name, proposed_activation in self._generate_activations().items():
            proposed_payload = config_tree.clean_config(proposed_activation)
            proposed_paused = proposed_payload['paused']
            if runner_name not in self._gitlab_runner_overview:
                print(f'Skipping {runner_name}, not found')
                continue
            current_runner = self._gitlab_runner_overview[runner_name]
            if current_runner['paused'] != proposed_paused:
                print(f'updating "paused" of {runner_name} to {proposed_paused}',
                      file=sys.stderr)
                if not self.args.dry_run:
                    gl_instance = self._gl_instance(current_runner['.instance_name'])
                    gl_instance.runners.update(current_runner['id'], proposed_payload)

    def _dump_variables(self):
        """Perform a 'variables dump' command."""
        groups = {}
        projects = {}
        for item in self.gitlab_variables:
            groups.update({g: self._gitlab_variables(self._gl_group(g))
                           for g in self._list(item, 'gitlab_groups')})
            projects.update({p: self._gitlab_variables(self._gl_project(p))
                             for p in self._list(item, 'gitlab_projects')})
        return self._sorted({'groups': groups, 'projects': projects})

    def _generate_variables(self):
        """Perform a 'variables generate' command."""
        projects = {}
        groups = {}
        for item in self.gitlab_variables:
            variables = self.variable_groups[item['variable_group']]
            groups.update({g: copy.deepcopy(variables)
                           for g in self._list(item, 'gitlab_groups')})
            projects.update({p: copy.deepcopy(variables)
                             for p in self._list(item, 'gitlab_projects')})
        return self._sorted({'groups': groups, 'projects': projects})

    def _apply_variable(self, name, gl_project_group,
                        current_variables, proposed_variables):
        """Perform part of a 'variables apply' command."""
        for key in set(current_variables) - set(proposed_variables):
            print(f'{name}: deleting variable {key}', file=sys.stderr)
            if not self.args.dry_run:
                gl_project_group.variables.delete(key)
        for key, value in proposed_variables.items():
            payload = {'key': key, 'value': value,
                       'variable_type': 'env_var'}
            if key not in current_variables:
                print(f'{name}: creating variable {key}', file=sys.stderr)
                if not self.args.dry_run:
                    try:
                        gl_project_group.variables.create({**payload, 'masked': True})
                    # pylint: disable=broad-exception-caught
                    except Exception:
                        gl_project_group.variables.create(payload)
            elif current_variables[key] != value:
                print(f'{name}: updating variable {key}', file=sys.stderr)
                if not self.args.dry_run:
                    gl_project_group.variables.update(key, payload)

    def _apply_variables(self):
        """Perform a 'variables apply' command."""
        current = self._dump_variables()
        proposed = self._generate_variables()

        if current == proposed:
            print('variables: nothing to do', file=sys.stderr)
            return

        for name, proposed_variables in proposed['groups'].items():
            gl_project_group = self._gl_group(name)
            current_variables = current['groups'][name]
            self._apply_variable(name, gl_project_group,
                                 current_variables, proposed_variables)
        for name, proposed_variables in proposed['projects'].items():
            gl_project_group = self._gl_project(name)
            current_variables = current['projects'][name]
            self._apply_variable(name, gl_project_group,
                                 current_variables, proposed_variables)

    def _instance_for_routing_key(self, routing_key):
        """Try to find the GitLab instance for a routing key.

        Returns the instance abbreviation. In case no instance could be found,
        returns None.
        """
        for instance_name, instance in self.gitlab_instances.items():
            if routing_key.startswith(urlparse(instance['url']).hostname):
                return instance_name
        return None

    @staticmethod
    def _webhook_from_group(instance_name, gl_instance, group):
        print(f'Resolving group for {group}', file=sys.stderr)
        with contextlib.suppress(GitlabGetError):
            gl_group = gl_instance.groups.get(re.sub('/[^/]*[#*].*', '', group.replace('.', '/')))
            return (f'{instance_name}/{gl_group.full_path}',
                    'subgroups' if '#' in group else 'group')

        raise Exception(f'Unable to find group for {group}')

    @staticmethod
    def _webhook_from_project(instance_name, gl_instance, project):
        print(f'Resolving project for {project}', file=sys.stderr)
        # the simple case first
        with contextlib.suppress(GitlabGetError):
            gl_project = gl_instance.projects.get(project.replace('.', '/'))
            return (f'{instance_name}/{gl_project.path_with_namespace}', 'project')
        # look through groups
        groups = {project[len(g.full_path)+1:]: g.full_path
                  for g in gl_instance.groups.list(iterator=True)
                  if project.startswith(g.full_path.replace('/', '.'))}
        for name, path in groups.items():
            with contextlib.suppress(GitlabGetError):
                gl_project = gl_instance.projects.get(path + '/' + name)
                return (f'{instance_name}/{gl_project.path_with_namespace}', 'project')

        raise Exception(f'Unable to find project for {project}')

    def _webhook_for_routing_key(self, routing_key):
        """Try to find the group/project for a routing key.

        Returns a tuple of short group/project name and type.

        If the group/project cannot be found, raises an exception.

        Do not use directly, use webhook_for_routing_key() which caches
        groups/projects.
        """
        instance_name = self._instance_for_routing_key(routing_key)
        gl_instance = self._gl_instance(instance_name)
        instance_url = self.gitlab_instances[instance_name]['url']
        project_group = routing_key[len(urlparse(instance_url).hostname)+1:]

        # wildcards are only supported for simple groups
        if any(w in project_group for w in '*#'):
            return self._webhook_from_group(instance_name, gl_instance, project_group)
        return self._webhook_from_project(instance_name, gl_instance, project_group)

    def _webhook_configs(self, routing_keys):
        """Return a list of group/project names and events for the routing keys."""
        flat_routing_key_events = [r.rsplit('.', 1)
                                   for rs in routing_keys for r in rs.split()]
        return [(*self.webhook_for_routing_key(r[0]), r[1])
                for r in flat_routing_key_events
                if self._instance_for_routing_key(r[0])]

    @staticmethod
    def _remove_duplicated_webhook_events(proposed):
        """Remove events from project hooks that are covered by group hooks."""
        # reshuffle the webhooks to be indexed by hook.project instead of the other way round
        hook_urls = set(sum((list(v.keys()) for v in proposed.values()), []))
        projects_by_hook = {hook_url: {project_name: project_hooks[hook_url]
                                       for project_name, project_hooks in proposed.items()
                                       if hook_url in project_hooks}
                            for hook_url in hook_urls}
        # avoid duplication of events by selectively disabling project events
        for hook_url, hook_configs in projects_by_hook.items():
            for group_key, group_hook in hook_configs.items():
                if 'subgroup_events' not in group_hook:
                    continue
                enabled_group_events = [
                    k for k, v in group_hook.items()
                    if k.endswith('_events') and k != 'subgroup_events' and v]
                for affected_key, affected_hook in hook_configs.items():
                    if affected_key == group_key or not affected_key.startswith(group_key):
                        continue
                    partial_key = affected_key[len(group_key) + 1:]
                    if '/' in partial_key and not group_hook['subgroup_events']:
                        continue
                    for name in enabled_group_events:
                        affected_hook[name] = False
        return proposed

    def _webhooks_endpoints(self):
        """Return data on all requested webhook endpoints."""
        proposed = {}
        for item in self.webhook_endpoints:
            configs = self._webhook_configs(self._list(item, '.routing_keys'))
            for name, hook_type, event in configs:
                project_group = proposed.setdefault(name, {})
                hook = project_group.setdefault(item['url'], {
                    'url': item['url'],
                    'enable_ssl_verification': item.get(
                        'enable_ssl_verification', True),
                    'confidential_issues_events': False,
                    'deployment_events': False,
                    'issues_events': False,
                    'job_events': False,
                    'merge_requests_events': False,
                    'note_events': False,
                    'pipeline_events': False,
                    'push_events': False,
                    '.pattern': item.get('.pattern', re.escape(item['url'])),
                    '.secret_token': item['.secret_token'],
                })
                if hook_type == 'subgroups':
                    hook['subgroup_events'] = True
                elif hook_type == 'group':
                    hook['subgroup_events'] = False
                if item.get('.active', True):
                    if event == 'build':
                        hook['job_events'] = True
                    elif event == 'merge_request':
                        hook['merge_requests_events'] = True
                    elif event == 'issue':
                        hook['confidential_issues_events'] = True
                        hook['issues_events'] = True
                    elif event in ('note', 'pipeline', 'push', 'deployment'):
                        hook[f'{event}_events'] = True
                    else:
                        raise Exception(f'Unknown event type {event}')
        return self._remove_duplicated_webhook_events(proposed)

    @staticmethod
    def _current_webhook(project_group, gl_project_group, proposed_hook):
        """Try to find an existing hook.

        Returns the gitlab hook instance.
        """
        for gl_hook in gl_project_group.hooks.list(iterator=True):
            if re.fullmatch(proposed_hook['.pattern'], gl_hook.url):
                if 'disabled' in gl_hook.attributes.get('alert_status', ''):
                    LOGGER.error('Hook %s at %s is disabled!',
                                 proposed_hook['.pattern'], project_group)
                return gl_hook
        return None

    @staticmethod
    def _current_webhook_data(gl_hook):
        """Return the filtered data from a gitlab hook instance."""
        return {k: v for k, v in gl_hook.attributes.items()
                if k in ('url', 'enable_ssl_verification',
                         'subgroup_events', 'confidential_issues_events'
                         'deployment_events', 'issues_events',
                         'job_events', 'merge_requests_events',
                         'note_events', 'pipeline_events', 'push_events')
                }

    def _dump_webhooks(self):
        """Perform a 'webhooks dump' command."""
        current = {}
        proposed = self._webhooks_endpoints()
        for project_group, proposed_hooks in proposed.items():
            if 'subgroup_events' in next(iter(proposed_hooks.values())):
                gl_project_group = self._gl_group(project_group)
            else:
                gl_project_group = self._gl_project(project_group)
            for hook_name, proposed_hook in proposed_hooks.items():
                gl_hook = self._current_webhook(project_group, gl_project_group, proposed_hook)
                if not gl_hook:
                    continue
                current_hooks = current.setdefault(project_group, {})
                current_hooks[hook_name] = self._current_webhook_data(gl_hook)
        return self._sorted(current)

    def _generate_webhooks(self):
        """Perform a 'webhooks generate' command."""
        return self._sorted(config_tree.clean_config(self._webhooks_endpoints()))

    def _apply_webhook(self, project_group, name, proposed_hook):
        """Apply changes to one webhook."""
        if 'subgroup_events' in proposed_hook:
            gl_project_group = self._gl_group(project_group)
        else:
            gl_project_group = self._gl_project(project_group)
        proposed_payload = config_tree.clean_config(proposed_hook)
        gl_hook = self._current_webhook(project_group, gl_project_group, proposed_hook)
        if not gl_hook:
            print(f'{project_group}: creating hook {name}', file=sys.stderr)
            proposed_payload['token'] = proposed_hook['.secret_token']
            if not self.args.dry_run:
                gl_project_group.hooks.create(proposed_payload)
        else:
            current_payload = self._current_webhook_data(gl_hook)
            if current_payload != proposed_payload:
                print(f'{project_group}: updating hook {name}', file=sys.stderr)
                proposed_payload['token'] = proposed_hook['.secret_token']
                if not self.args.dry_run:
                    gl_project_group.hooks.update(gl_hook.id, proposed_payload)

    def _apply_webhooks(self):
        """Perform a 'webhooks apply' command."""
        proposed = self._webhooks_endpoints()
        for project_group, proposed_hooks in proposed.items():
            for hook_name, proposed_hook in proposed_hooks.items():
                self._apply_webhook(project_group, hook_name, proposed_hook)

    def dump(self, what):
        """Dump current values."""
        return yaml.dump(getattr(self, f'_dump_{what}')())

    def generate(self, what):
        """Generate proposed values."""
        return yaml.dump(getattr(self, f'_generate_{what}')())

    def diff(self, what):
        """Diff current and proposed values."""
        current = self.dump(what)
        proposed = self.generate(what)

        if current == proposed:
            print(f'{what}: nothing to do', file=sys.stderr)
            return None

        if self.args.interactive:
            with self._tempfile('current', current) as name1, \
                    self._tempfile('proposed', proposed) as name2:
                subprocess.run(['vimdiff', name1, name2], check=False)
            return None

        return '\n'.join(difflib.unified_diff(
            current.split('\n'), proposed.split('\n'),
            fromfile='current', tofile='proposed', n=self.args.context))

    def apply(self, what):
        """Apply the proposed values."""
        return getattr(self, f'_apply_{what}')()


def main(argv):
    """Command line interface to the gitlab-runner config management."""
    objects = ['registrations', 'activations', 'configurations', 'variables', 'webhooks']
    actions = ['dump', 'generate', 'diff', 'apply']
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        '-c', '--config', default='runners.yaml', type=argparse.FileType('r'),
        help='YAML configuration file to use')
    parser.add_argument(
        '--activate', metavar='REGEX', default=[], action='append',
        help='unconditionally activate matching runners')
    parser.add_argument(
        '--deactivate', metavar='REGEX', default=[], action='append',
        help='unconditionally deactivate matching runners, processed after --activate')
    parser.add_argument(
        '--directory', default=os.getcwd(),
        help='directory for relative file names')
    parser.add_argument(
        'object', choices=objects,
        help='configuration object')
    parser.add_argument(
        'action', choices=actions,
        help='configuration action')

    diff_args = parser.add_argument_group('diff action')
    diff_args.add_argument(
        '--context', default=10, type=int,
        help='number of context lines')
    diff_args.add_argument(
        '-i', '--interactive', action='store_true',
        help='show the differences with vimdiff')

    apply_args = parser.add_argument_group('apply action')
    apply_args.add_argument(
        '--dry-run', action='store_true',
        help='Only print the actions that would be taken, do not take them')
    apply_args.add_argument(
        '--create-missing', action='store_true',
        help='Also perform new runner registrations')

    args = parser.parse_args(argv)

    with args.config as config_file:
        config = yaml.load(contents=config_file.read())
    runner_config = RunnerConfig(config, args)
    result = getattr(runner_config, args.action)(args.object)
    if result:
        print(result)
    return result


if __name__ == '__main__':
    misc.sentry_init(sentry_sdk)
    main(sys.argv[1:])
